#!/usr/bin/env python
#-*- coding:utf-8 -*-

import urlparse
import hashlib
import re
from pybloomfilter import BloomFilter

class uniqueUrl():
    def __init__(self):
        self.__bf = BloomFilter(10000000, 0.01, './data/filter.bloom')

    def add(self, url):
        self.__url = url
        matchObj = re.search(r'(.exe|.jpg|.jepg|.gif|.png|.swf|.js|.css|.txt|.doc|.docx|.xls|.xlsx|.ppt|.pdf|.zip|.rar|.7z|.ico|.mp4|.avi|.rm|.rmvb|.3gp|.mkv|.mp3|.wma)$',self.__url,re.I)
        if matchObj:
            return True
        elif self.__bf.add(self.__createurlhash(self.__url)):
            return True
        else:
            return False

    #Create unique hash
    def __createurlhash(self, url):
        parse = urlparse.urlparse(url)
        urlquery = parse.query

        if len(urlquery) == 0:
            #print '[+] Url [%s] have no query string.' % url
            return hashlib.md5(url).hexdigest()

        querysplit = urlquery.split('&')
        querysort = sorted(querysplit)

        newquery = list()

        for param in querysort:
            tmp = param.split('=')
            if len(tmp[1]) == 0:
                tmp[1] = 'null'
            elif tmp[1].isdigit():
                tmp[1] = '%d'
            else:
                tmp[1] = '%s'

            newquery.append('='.join(tmp))

        newquery = '&'.join(newquery)
        newurl = urlparse.urlunparse((parse.scheme,parse.netloc, parse.path, parse.params, newquery, parse.fragment))

        return hashlib.md5(newurl).hexdigest()

if __name__ == '__main__':

    url = ['http://www.baidu.com/index.php?type=866&id=153', 
    'http://www.baidu.com/index.php?type=789&id=33', 
    'http://www.baidu.com/index.php?type=789&id=we',
    'http://www.baidu.com/help.jpg']

    #init uniqueUrl
    u = uniqueUrl()

    for t in url:
       
        if u.add(t):
            print '%s \t Found in list' % t
        else:
            print '%s \t Not found' % t